{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b0883de1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "da3f26a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d=np.ones((5,3,3))\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9af14af3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0., 1., 1.],\n",
       "        [0., 0., 1.],\n",
       "        [0., 0., 0.]],\n",
       "\n",
       "       [[0., 1., 1.],\n",
       "        [0., 0., 1.],\n",
       "        [0., 0., 0.]],\n",
       "\n",
       "       [[0., 1., 1.],\n",
       "        [0., 0., 1.],\n",
       "        [0., 0., 0.]],\n",
       "\n",
       "       [[0., 1., 1.],\n",
       "        [0., 0., 1.],\n",
       "        [0., 0., 0.]],\n",
       "\n",
       "       [[0., 1., 1.],\n",
       "        [0., 0., 1.],\n",
       "        [0., 0., 0.]]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.triu(d,k=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "40c69713",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]],\n",
       "\n",
       "        [[1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]],\n",
       "\n",
       "        [[1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]],\n",
       "\n",
       "        [[1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]],\n",
       "\n",
       "        [[1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 1., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 1., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 1., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 1., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 1., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 1., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 1., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 1., 1.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask=torch.ones(5,10,10)\n",
    "mask = torch.triu(mask,diagonal=0)\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8f9d5642",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95a0e718",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9fe30a9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.720075976020836e-44"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "math.exp(-100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "668222a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=[[ 3.4108e+01,  1.3441e+00,  7.9152e-01,  1.0932e+00,  8.0256e-01,\n",
    "          -1.1138e+00,  5.2862e-01,  1.2744e+00,  2.1217e+00, -7.9262e-01],\n",
    "         [ 1.3441e+00,  3.6377e+01,  1.9868e+00, -1.7119e-01, -2.0109e+00,\n",
    "          -3.5834e-01,  5.1095e-01,  2.5595e-01, -7.4656e-01,  2.7426e-01],\n",
    "         [ 7.9152e-01,  1.9868e+00,  3.1066e+01,  2.0734e+01,  8.3555e-01,\n",
    "           1.0830e+00,  2.4475e+00,  1.1955e+00,  1.6111e+00, -1.0380e+00],\n",
    "         [ 1.0932e+00, -1.7119e-01,  2.0734e+01,  3.3057e+01,  1.5844e-01,\n",
    "          -7.0308e-01,  2.0422e+00,  8.5928e-01,  2.9717e+00, -1.2110e+00],\n",
    "         [ 8.0256e-01, -2.0109e+00,  8.3555e-01,  1.5844e-01,  3.2980e+01,\n",
    "           3.5099e-01,  3.9910e-01, -6.1639e-01,  2.8646e+00, -1.2619e-01],\n",
    "         [-1.1138e+00, -3.5834e-01,  1.0830e+00, -7.0308e-01,  3.5099e-01,\n",
    "           2.9563e+01,  1.5085e+00, -1.0889e+00,  2.1407e+00, -2.0102e+00],\n",
    "         [ 5.2862e-01,  5.1095e-01,  2.4475e+00,  2.0422e+00,  3.9910e-01,\n",
    "           1.5085e+00,  3.3341e+01, -1.0539e+00, -1.2324e+00,  3.9669e-02],\n",
    "         [ 1.2744e+00,  2.5595e-01,  1.1955e+00,  8.5928e-01, -6.1639e-01,\n",
    "          -1.0889e+00, -1.0539e+00,  3.2315e+01, -6.6442e-01, -2.3448e+00],\n",
    "         [ 2.1217e+00, -7.4656e-01,  1.6111e+00,  2.9717e+00,  2.8646e+00,\n",
    "           2.1407e+00, -1.2324e+00, -6.6442e-01,  3.4236e+01, -1.3915e+00],\n",
    "         [-7.9262e-01,  2.7426e-01, -1.0380e+00, -1.2110e+00, -1.2619e-01,\n",
    "          -2.0102e+00,  3.9669e-02, -2.3448e+00, -1.3915e+00,  3.5494e+01]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b779a499",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[34.108,\n",
       "  1.3441,\n",
       "  0.79152,\n",
       "  1.0932,\n",
       "  0.80256,\n",
       "  -1.1138,\n",
       "  0.52862,\n",
       "  1.2744,\n",
       "  2.1217,\n",
       "  -0.79262],\n",
       " [1.3441,\n",
       "  36.377,\n",
       "  1.9868,\n",
       "  -0.17119,\n",
       "  -2.0109,\n",
       "  -0.35834,\n",
       "  0.51095,\n",
       "  0.25595,\n",
       "  -0.74656,\n",
       "  0.27426],\n",
       " [0.79152,\n",
       "  1.9868,\n",
       "  31.066,\n",
       "  20.734,\n",
       "  0.83555,\n",
       "  1.083,\n",
       "  2.4475,\n",
       "  1.1955,\n",
       "  1.6111,\n",
       "  -1.038],\n",
       " [1.0932,\n",
       "  -0.17119,\n",
       "  20.734,\n",
       "  33.057,\n",
       "  0.15844,\n",
       "  -0.70308,\n",
       "  2.0422,\n",
       "  0.85928,\n",
       "  2.9717,\n",
       "  -1.211],\n",
       " [0.80256,\n",
       "  -2.0109,\n",
       "  0.83555,\n",
       "  0.15844,\n",
       "  32.98,\n",
       "  0.35099,\n",
       "  0.3991,\n",
       "  -0.61639,\n",
       "  2.8646,\n",
       "  -0.12619],\n",
       " [-1.1138,\n",
       "  -0.35834,\n",
       "  1.083,\n",
       "  -0.70308,\n",
       "  0.35099,\n",
       "  29.563,\n",
       "  1.5085,\n",
       "  -1.0889,\n",
       "  2.1407,\n",
       "  -2.0102],\n",
       " [0.52862,\n",
       "  0.51095,\n",
       "  2.4475,\n",
       "  2.0422,\n",
       "  0.3991,\n",
       "  1.5085,\n",
       "  33.341,\n",
       "  -1.0539,\n",
       "  -1.2324,\n",
       "  0.039669],\n",
       " [1.2744,\n",
       "  0.25595,\n",
       "  1.1955,\n",
       "  0.85928,\n",
       "  -0.61639,\n",
       "  -1.0889,\n",
       "  -1.0539,\n",
       "  32.315,\n",
       "  -0.66442,\n",
       "  -2.3448],\n",
       " [2.1217,\n",
       "  -0.74656,\n",
       "  1.6111,\n",
       "  2.9717,\n",
       "  2.8646,\n",
       "  2.1407,\n",
       "  -1.2324,\n",
       "  -0.66442,\n",
       "  34.236,\n",
       "  -1.3915],\n",
       " [-0.79262,\n",
       "  0.27426,\n",
       "  -1.038,\n",
       "  -1.211,\n",
       "  -0.12619,\n",
       "  -2.0102,\n",
       "  0.039669,\n",
       "  -2.3448,\n",
       "  -1.3915,\n",
       "  35.494]]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1d87b1b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6.50004239e+14, 3.83473373e+00, 2.20674814e+00, 2.98380699e+00,\n",
       "        2.23124561e+00, 3.28309013e-01, 1.69658940e+00, 3.57655483e+00,\n",
       "        8.34531247e+00, 4.52657278e-01],\n",
       "       [3.83473373e+00, 6.28536305e+15, 7.29216147e+00, 8.42661453e-01,\n",
       "        1.33868139e-01, 6.98835431e-01, 1.66687397e+00, 1.29168814e+00,\n",
       "        4.73994292e-01, 1.31555680e+00],\n",
       "       [2.20674814e+00, 7.29216147e+00, 3.10307573e+13, 1.01079198e+09,\n",
       "        2.30608204e+00, 2.95352685e+00, 1.15594120e+01, 3.30520996e+00,\n",
       "        5.00831735e+00, 3.54162299e-01],\n",
       "       [2.98380699e+00, 8.42661453e-01, 1.01079198e+09, 2.27233673e+14,\n",
       "        1.17168162e+00, 4.95058174e-01, 7.70754716e+00, 2.36145983e+00,\n",
       "        1.95250840e+01, 2.97899231e-01],\n",
       "       [2.23124561e+00, 1.33868139e-01, 2.30608204e+00, 1.17168162e+00,\n",
       "        2.10393352e+14, 1.42047312e+00, 1.49048266e+00, 5.39889927e-01,\n",
       "        1.75420350e+01, 8.81447356e-01],\n",
       "       [3.28309013e-01, 6.98835431e-01, 2.95352685e+00, 4.95058174e-01,\n",
       "        1.42047312e+00, 6.90315728e+12, 4.51994579e+00, 3.36586535e-01,\n",
       "        8.50538932e+00, 1.33961880e-01],\n",
       "       [1.69658940e+00, 1.66687397e+00, 1.15594120e+01, 7.70754716e+00,\n",
       "        1.49048266e+00, 4.51994579e+00, 3.01864694e+14, 3.48575650e-01,\n",
       "        2.91591917e-01, 1.04046632e+00],\n",
       "       [3.57655483e+00, 1.29168814e+00, 3.30520996e+00, 2.36145983e+00,\n",
       "        5.39889927e-01, 3.36586535e-01, 3.48575650e-01, 1.08199731e+14,\n",
       "        5.14571893e-01, 9.58663735e-02],\n",
       "       [8.34531247e+00, 4.73994292e-01, 5.00831735e+00, 1.95250840e+01,\n",
       "        1.75420350e+01, 8.50538932e+00, 2.91591917e-01, 5.14571893e-01,\n",
       "        7.38764269e+14, 2.48701972e-01],\n",
       "       [4.52657278e-01, 1.31555680e+00, 3.54162299e-01, 2.97899231e-01,\n",
       "        8.81447356e-01, 1.33961880e-01, 1.04046632e+00, 9.58663735e-02,\n",
       "        2.48701972e-01, 2.59925172e+15]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.exp(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3fba325f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "650004238781968.8"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "math.exp(34.108)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6b92cc69",
   "metadata": {},
   "outputs": [],
   "source": [
    "q=k=v=torch.randn(1,5,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3cfa2029",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.8457,  0.0082,  0.5799,  0.9960],\n",
       "         [ 1.5332, -1.3987,  1.2019,  0.6469],\n",
       "         [ 0.2561,  0.5535,  1.8772, -0.6888],\n",
       "         [ 0.5201, -0.2154,  0.3361,  0.8580],\n",
       "         [ 0.6304,  0.1523, -0.7619, -2.9101]]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "02334bea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 2.0437,  0.0331,  0.1904,  0.6078, -3.8723],\n",
       "         [ 0.0331,  6.1702,  1.4291,  2.0577, -2.0449],\n",
       "         [ 0.1904,  1.4291,  4.3702,  0.0539,  0.8201],\n",
       "         [ 0.6078,  2.0577,  0.0539,  1.1660, -2.4578],\n",
       "         [-3.8723, -2.0449,  0.8201, -2.4578,  9.4699]]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(q,k.transpose(-2,-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5be3a4b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.8457,  1.5332,  0.2561,  0.5201,  0.6304],\n",
       "         [ 0.0082, -1.3987,  0.5535, -0.2154,  0.1523],\n",
       "         [ 0.5799,  1.2019,  1.8772,  0.3361, -0.7619],\n",
       "         [ 0.9960,  0.6469, -0.6888,  0.8580, -2.9101]]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k.transpose(-2,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cd8d2aa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caf68203",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e16b4b76",
   "metadata": {},
   "outputs": [],
   "source": [
    "a=torch.arange(0,10).unsqueeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5e83ca9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "b=torch.arange(0,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "238bde8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9],\n",
       "        [ 0,  2,  4,  6,  8, 10, 12, 14, 16, 18],\n",
       "        [ 0,  3,  6,  9, 12, 15, 18, 21, 24, 27],\n",
       "        [ 0,  4,  8, 12, 16, 20, 24, 28, 32, 36],\n",
       "        [ 0,  5, 10, 15, 20, 25, 30, 35, 40, 45],\n",
       "        [ 0,  6, 12, 18, 24, 30, 36, 42, 48, 54],\n",
       "        [ 0,  7, 14, 21, 28, 35, 42, 49, 56, 63],\n",
       "        [ 0,  8, 16, 24, 32, 40, 48, 56, 64, 72],\n",
       "        [ 0,  9, 18, 27, 36, 45, 54, 63, 72, 81]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a*b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c583907",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0.],\n",
       "        [0., 0., 0.],\n",
       "        [0., 0., 0.]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "35c96e76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0.],\n",
       "        [0., 0., 0.],\n",
       "        [0., 0., 0.]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.triu(torch.zeros(3,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fb2d2973",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0.],\n",
       "        [0., 0., 0.],\n",
       "        [0., 0., 0.]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.zeros(3,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "499c9884",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'np' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mnp\u001b[49m\u001b[38;5;241m.\u001b[39mzeros(\u001b[38;5;241m2\u001b[39m,\u001b[38;5;241m3\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'np' is not defined"
     ]
    }
   ],
   "source": [
    "np.zeros(2,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6519da0f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "568939d7",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.ndarray' object has no attribute 'unseqeze'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[6], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marange\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munseqeze\u001b[49m(\u001b[38;5;241m1\u001b[39m)\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'unseqeze'"
     ]
    }
   ],
   "source": [
    "np.arange(0,10).unseqeze(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bf9b428c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0],\n",
       "        [1],\n",
       "        [2],\n",
       "        [3],\n",
       "        [4]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b=torch.arange(0,5).unsqueeze(1)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cae067fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0,  0,  0,  0],\n",
       "        [ 0,  2,  4,  6,  8],\n",
       "        [ 0,  4,  8, 12, 16],\n",
       "        [ 0,  6, 12, 18, 24],\n",
       "        [ 0,  8, 16, 24, 32]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b*a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ad9d3693",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.8645, -0.2632,  1.5102,  0.2223, -2.0941],\n",
       "         [-0.2881,  3.1751,  1.7617,  0.2247, -0.6307],\n",
       "         [-0.0162, -0.5065, -0.9981, -1.0635, -1.5912],\n",
       "         [-1.5924, -0.1541, -1.0626, -0.0823, -0.7153],\n",
       "         [-0.9243,  0.3751,  0.5408, -0.5589,  0.1030]],\n",
       "\n",
       "        [[-0.2711, -0.4054,  0.6658,  0.8601, -1.0091],\n",
       "         [ 1.5418, -0.1633,  0.9117, -1.8298, -0.6745],\n",
       "         [-0.7382,  0.3780,  1.3314, -1.2396,  2.1954],\n",
       "         [ 1.9809,  0.4448, -0.5120,  1.1294,  1.4526],\n",
       "         [ 1.6221, -0.5176, -1.3468,  1.1331,  2.1846]]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.randn((2,5,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "12a0ca0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = np.random.randn(9,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9e32dded",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.08677387,  0.59222774],\n",
       "       [-0.06246107,  0.29944815],\n",
       "       [ 0.57123286, -1.73227911],\n",
       "       [-1.26980255,  0.29243162],\n",
       "       [ 0.10762824,  1.56435753],\n",
       "       [ 0.58221899,  0.09072691],\n",
       "       [-0.82496783,  1.40184878],\n",
       "       [-0.2564474 , -1.46391922],\n",
       "       [ 1.19214961,  0.89820033]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c4a2be1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.08677387,  0.59222774],\n",
       "       [ 0.57123286, -1.73227911],\n",
       "       [ 0.10762824,  1.56435753],\n",
       "       [-0.82496783,  1.40184878],\n",
       "       [ 1.19214961,  0.89820033]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[0::2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "46ef7751",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.06246107,  0.29944815],\n",
       "       [-1.26980255,  0.29243162],\n",
       "       [ 0.58221899,  0.09072691],\n",
       "       [-0.2564474 , -1.46391922]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[1::2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b419d680",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76ac1a19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [1.],\n",
       "        [2.],\n",
       "        [3.],\n",
       "        [4.],\n",
       "        [5.],\n",
       "        [6.],\n",
       "        [7.],\n",
       "        [8.],\n",
       "        [9.]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.arange(0, 10, dtype=torch.float).unsqueeze(1)\n",
    "a  # 10,1\n",
    "b = torch.arange(0, 5)\n",
    "b # 5\n",
    "a*b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b8b6580",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eb76d012",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "50326b2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-9.210340371976184"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-math.log(10000.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "50960303",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'torch' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[5], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241m.\u001b[39marange(\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m10\u001b[39m,\u001b[38;5;241m2\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'torch' is not defined"
     ]
    }
   ],
   "source": [
    "torch.arange(0,10,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "65183db7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0000, 0.0050, 0.0100, 0.0150, 0.0200],\n",
      "        [0.0000, 0.0050, 0.0100, 0.0150, 0.0200],\n",
      "        [0.0000, 0.0050, 0.0100, 0.0150, 0.0200],\n",
      "        [0.0000, 0.0050, 0.0100, 0.0150, 0.0200],\n",
      "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000]])\n",
      "tensor([-0.0065, -0.0071, -0.0103,  0.0088,  0.0106])\n"
     ]
    }
   ],
   "source": [
    "a = torch.arange(0, 4, dtype=torch.long).unsqueeze(1)\n",
    "\n",
    "emb = nn.Embedding(10, 5)\n",
    "a = emb(a)/10\n",
    "\n",
    "# a.requires_grad_(True)\n",
    "a  # 10,1\n",
    "b = torch.arange(0, 5,dtype=torch.float)\n",
    "b.requires_grad_(True)\n",
    "b # 5\n",
    "loss=torch.mean(a*b)\n",
    "loss.backward()\n",
    "\n",
    "for x in emb.parameters():\n",
    "    print(x.grad)\n",
    "\n",
    "print(b.grad)\n",
    "# display(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "f61d4b4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "bc5990ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "b.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "abf80c16",
   "metadata": {},
   "outputs": [],
   "source": [
    "a.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "025666d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 5])\n"
     ]
    }
   ],
   "source": [
    "for x in emb.parameters():\n",
    "    print(x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a56926a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch11.3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
